{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorflow'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-92e58a81a18c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mtensorflow\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'tensorflow'"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import random\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 定义读取数据的类,像mnist那样\n",
    "class WSTrainData(object):\n",
    "    def __init__(self, data_folder):\n",
    "        self.data_folder = data_folder\n",
    "        self.features = []\n",
    "        self.labels = []\n",
    "        \n",
    "        self._query_data()\n",
    "        print(\"features: %d, labels: %d\" % (len(self.features), len(self.labels)))\n",
    "    \n",
    "    def _query_data(self):\n",
    "        print(self.data_folder)\n",
    "        for root, dirs, files in os.walk(self.data_folder):\n",
    "            for ifile in files:\n",
    "                cur_file = os.path.join(root, ifile)\n",
    "                cur_features, cur_labels = self._query_file_data(cur_file)\n",
    "                self.features.extend(cur_features)\n",
    "                self.labels.extend(cur_labels)\n",
    "                \n",
    "    def _query_file_data(self, file_name):\n",
    "        features = []\n",
    "        labels = []\n",
    "        with open(file_name, 'r', encoding=\"utf-8-sig\") as pfile:\n",
    "            for isent in [x.strip() for x in pfile if x.strip() != \"\"]:\n",
    "                words = isent.split()\n",
    "                chars = []\n",
    "                tags = []\n",
    "                for iword in words:\n",
    "                    chars.extend(list(iword))\n",
    "                    if len(iword) == 1:\n",
    "                        tags.append(\"S\")\n",
    "                    else:\n",
    "                        tags.extend([\"B\"]+[\"M\"]*(len(iword)-2)+[\"E\"])\n",
    "                features.append(chars)\n",
    "                labels.append(tags)\n",
    "        \n",
    "        return features, labels\n",
    "    \n",
    "    def next_batch(self, batch_size):\n",
    "        return list(zip(*random.sample(list(zip(self.features, self.labels)), batch_size)))\n",
    "    \n",
    "    def get_data(self):\n",
    "        return self.features, self.labels\n",
    "    \n",
    "\n",
    "# class WSDataReader(object):\n",
    "#     def __init__(self, train_folder, test_folder, dev_folder):\n",
    "#         self.train_data = "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建数据转换等工具函数\n",
    "def create_dic(item_list, add_unk=False, add_pad=False):\n",
    "    \"\"\"\n",
    "    Create a dictionary of items from a list of list of items.\n",
    "    \"\"\"\n",
    "    assert type(item_list) in (list, tuple)\n",
    "    dic = {}\n",
    "    for items in item_list:\n",
    "        for item in items:\n",
    "            if item not in dic:\n",
    "                dic[item] = 1\n",
    "            else:\n",
    "                dic[item] += 1\n",
    "    # Make sure that <PAD> have a id 0.\n",
    "    if add_pad:\n",
    "        dic['<PAD>'] = 1e20\n",
    "    # If specified, add a special item <UNK>.\n",
    "    if add_unk:\n",
    "        dic['<UNK>'] = 1e10\n",
    "    return dic\n",
    "\n",
    "def create_mapping(items):\n",
    "    \"\"\"\n",
    "    Create a mapping (item to ID / ID to item) from a dictionary.\n",
    "    Items are ordered by decreasing frequency.\n",
    "    \"\"\"\n",
    "    if type(items) is dict:\n",
    "        sorted_items = sorted(items.items(), key=lambda x: (-x[1], x[0]))\n",
    "        id2item = {i: v[0] for i, v in enumerate(sorted_items)}\n",
    "        item2id = {v: k for k, v in id2item.items()}\n",
    "        return item2id, id2item\n",
    "    elif type(items) is list:\n",
    "        id2item = {i: v for i, v in enumerate(items)}\n",
    "        item2id = {v: k for k, v in id2item.items()}\n",
    "        return item2id, id2item\n",
    "\n",
    "\n",
    "def create_input(batch):\n",
    "    \"\"\"\n",
    "    Take each sentence data in batch and return an input for\n",
    "    the training or the evaluation function.\n",
    "    \"\"\"\n",
    "    assert len(batch) > 0\n",
    "    lengths = [len(seq) for seq in batch]\n",
    "    max_len = max(2, max(lengths))\n",
    "    ret = []\n",
    "    for seq_id, pos in zip(batch, lengths):\n",
    "        assert len(seq_id) == pos\n",
    "        pad = [0] * (max_len - pos)\n",
    "        ret.append(np.array(seq_id + pad))\n",
    "    ret.append(lengths)\n",
    "    return ret\n",
    "\n",
    "def data_to_ids(data, mappings):\n",
    "    \"\"\"\n",
    "    Map text data to ids.\n",
    "    \"\"\"\n",
    "\n",
    "    def strQ2B(ustring):\n",
    "        rstring = \"\"\n",
    "        for uchar in ustring:\n",
    "            inside_code = ord(uchar)\n",
    "            if inside_code == 12288:\n",
    "                inside_code = 32\n",
    "            elif 65281 <= inside_code <= 65374:\n",
    "                inside_code -= 65248\n",
    "            rstring += chr(inside_code)\n",
    "        return rstring\n",
    "    def strB2Q(ustring):\n",
    "        rstring = \"\"\n",
    "        for uchar in ustring:\n",
    "            inside_code = ord(uchar)\n",
    "            if inside_code == 32:\n",
    "                inside_code = 12288\n",
    "            elif 32 <= inside_code <= 126:\n",
    "                inside_code += 65248\n",
    "            rstring += chr(inside_code)\n",
    "        return rstring\n",
    "\n",
    "    def map(item, mapping):\n",
    "        if item in mapping:\n",
    "            return mapping[item]\n",
    "        item = strB2Q(item)\n",
    "        if item in mapping:\n",
    "            return mapping[item]\n",
    "        item = strQ2B(item)\n",
    "\n",
    "    def map_seq(seqs, mapping):\n",
    "        return [[map(item, mapping) for item in seq] for seq in seqs]\n",
    "\n",
    "    ret = []\n",
    "    for d, m in zip(data, mappings):\n",
    "        ret.append(map_seq(d, m))\n",
    "    return tuple(ret)\n",
    "\n",
    "def data_iterator(inputs, batch_size, shuffle=True, max_length=200):\n",
    "    \"\"\"\n",
    "    A simple iterator for generating dynamic mini batches.\n",
    "    \"\"\"\n",
    "    assert len(inputs) > 0\n",
    "    assert all([len(item) == len(inputs[0]) for item in inputs])\n",
    "    inputs = zip(*inputs)\n",
    "    if shuffle:\n",
    "        np.random.shuffle(inputs)\n",
    "\n",
    "    batch = []\n",
    "    bs = batch_size\n",
    "    for d in inputs:\n",
    "        if len(d[0]) > max_length:\n",
    "            bs = max(1, min(batch_size * max_length / len(d[0]), bs))\n",
    "        if len(batch) < bs:\n",
    "            batch.append(d)\n",
    "        else:\n",
    "            yield zip(*batch)\n",
    "            batch = [d]\n",
    "            if len(d[0]) < max_length:\n",
    "                bs = batch_size\n",
    "            else:\n",
    "                bs = max(1, batch_size * max_length / len(d[0]))\n",
    "    if batch:\n",
    "        yield zip(*batch)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./datasets/sighan2005-pku/\n",
      "features: 0, labels: 0\n"
     ]
    }
   ],
   "source": [
    "train_data = WSTrainData(\"./datasets/sighan2005-pku/\")\n",
    "# 创建训练数据的字典,生成mapping,然后把训练数据都转为ID\n",
    "train_features_list, train_label_list = train_data.get_data()\n",
    "\n",
    "feature_dict = create_dic(train_features_list)\n",
    "label_dict = create_dic(train_label_list)\n",
    "\n",
    "feature_mapping = create_mapping(feature_dict)\n",
    "label_mapping = create_mapping(label_dict)\n",
    "\n",
    "feature_ids_list = data_to_ids(train_features_list, feature_mapping)\n",
    "label_ids_list = data_to_ids(train_label_list, label_mapping)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./data/data/train_pku\n",
      "features: 17149, labels: 17149\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0806 22:27:41.997025 140314993231680 deprecation.py:506] From /home/meixiao/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0806 22:27:42.422095 140314993231680 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0806 22:27:42.737175 140314993231680 deprecation.py:323] From <ipython-input-5-319fc748c938>:46: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See `tf.nn.softmax_cross_entropy_with_logits_v2`.\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "masks shape: (?, ?)\n",
      "output shape: (?, ?, 4)\n",
      "output shape: TensorShape([Dimension(None), Dimension(4)]), y_real shape: TensorShape([Dimension(None), Dimension(4)])\n",
      "(?, 4) (?, 4)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Cannot convert an unknown Dimension to a Tensor: ?",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-319fc748c938>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    106\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    107\u001b[0m \u001b[0mtrain_data_folder\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"./data/data/train_pku\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 108\u001b[0;31m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data_folder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-5-319fc748c938>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(data_folder)\u001b[0m\n\u001b[1;32m     72\u001b[0m     \u001b[0;31m# 创建网络\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m     \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membeddings\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membeddings_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_input_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfeature_dic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_size\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 74\u001b[0;31m     \u001b[0mcross_entropy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbuild_tag_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membeddings_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m512\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m128\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     75\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     76\u001b[0m     \u001b[0;31m# 创建优化方案\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-5-319fc748c938>\u001b[0m in \u001b[0;36mbuild_tag_graph\u001b[0;34m(inputs, hidden1_dim, hidden2_dim, num_classes, cur_label_ids_padding, lengths)\u001b[0m\n\u001b[1;32m     52\u001b[0m     \u001b[0mall_chars_num\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m     \u001b[0mall_true_chars_num\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_sum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlengths\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m     \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorrect_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdivide\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_true_chars_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mall_chars_num\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mcross_entropy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maccuracy\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/util/dispatch.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    178\u001b[0m     \u001b[0;34m\"\"\"Call target, and fall back on dispatchers if there is a TypeError.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    179\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 180\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    181\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mTypeError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m       \u001b[0;31m# Note: convert_to_eager_tensor currently raises a ValueError, not a\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mdivide\u001b[0;34m(x, y, name)\u001b[0m\n\u001b[1;32m    314\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mDivideDelegateWithName\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    315\u001b[0m   \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 316\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    317\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py\u001b[0m in \u001b[0;36mbinary_op_wrapper\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m           y = ops.convert_to_tensor_v2(\n\u001b[0;32m--> 888\u001b[0;31m               y, dtype_hint=x.dtype.base_dtype, name=\"y\")\n\u001b[0m\u001b[1;32m    889\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    890\u001b[0m           \u001b[0;31m# If the RHS is not a tensor, it might be a tensor aware object\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36mconvert_to_tensor_v2\u001b[0;34m(value, dtype, dtype_hint, name)\u001b[0m\n\u001b[1;32m   1143\u001b[0m       \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1144\u001b[0m       \u001b[0mpreferred_dtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype_hint\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1145\u001b[0;31m       as_ref=False)\n\u001b[0m\u001b[1;32m   1146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1147\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/ops.py\u001b[0m in \u001b[0;36minternal_convert_to_tensor\u001b[0;34m(value, dtype, name, as_ref, preferred_dtype, ctx, accept_symbolic_tensors, accept_composite_tensors)\u001b[0m\n\u001b[1;32m   1222\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1223\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1224\u001b[0;31m       \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconversion_func\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mas_ref\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mas_ref\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1225\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1226\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mret\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNotImplemented\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/mx_py3/lib/python3.7/site-packages/tensorflow/python/framework/constant_op.py\u001b[0m in \u001b[0;36m_dimension_tensor_conversion_function\u001b[0;34m(d, dtype, name, as_ref)\u001b[0m\n\u001b[1;32m    356\u001b[0m   \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mas_ref\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    357\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalue\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 358\u001b[0;31m     \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Cannot convert an unknown Dimension to a Tensor: %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    359\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    360\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdtypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtypes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint64\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Cannot convert an unknown Dimension to a Tensor: ?"
     ]
    }
   ],
   "source": [
    "# 数据处理完毕,接下来,定义模型结构:前向网络,损失函数,优化器,精度计算,运行网络\n",
    "num_classes = 4 # B,E,M,S\n",
    "\n",
    "# embedding 网络\n",
    "def build_input_graph(vocab_size, emb_size):\n",
    "    \"\"\"\n",
    "    transform to embeddings from lookup tables\n",
    "    \"\"\"\n",
    "    x = tf.placeholder(dtype=tf.int32, shape=[None, None])\n",
    "    y = tf.placeholder(dtype = tf.int32, shape=[None, None])\n",
    "    \n",
    "    embeddings = tf.get_variable(\"embeddings\", [vocab_size, emb_size])\n",
    "    embedding_output = tf.nn.embedding_lookup(embeddings, x)\n",
    "    \n",
    "    lengths = tf.placeholder(dtype=tf.int32, shape=[None])\n",
    "    \n",
    "    return x, y, embeddings, embedding_output, lengths\n",
    "\n",
    "# 简单的两层mlp，先跑通再说\n",
    "def build_tag_graph(inputs, hidden1_dim, hidden2_dim, num_classes, cur_label_ids_padding, lengths):\n",
    "    # inputs 已经是embedding, channels是embedding的维度\n",
    "    # 从Inputs中抽取出数据以及lengths\n",
    "    masks = tf.cast(tf.sequence_mask(lengths), tf.float32)\n",
    "    # 第一层fc，使用relu激活，\n",
    "    hidden1_output = tf.contrib.layers.fully_connected(inputs, hidden1_dim, tf.identity)\n",
    "    # 第二层\n",
    "    hidden2_output = tf.contrib.layers.fully_connected(hidden1_output, hidden2_dim, tf.identity)\n",
    "    # 第三层输出\n",
    "    output = tf.contrib.layers.fully_connected(hidden2_output, num_classes, tf.identity)\n",
    "    output = tf.nn.softmax(output)\n",
    "#     print(\"output shape: %s\" % output.shape)\n",
    "    \n",
    "    # 去掉padding\n",
    "    output = tf.multiply(output,  tf.expand_dims(masks, -1))\n",
    "    print(\"masks shape: %s\" % masks.shape)\n",
    "    print(\"output shape: %s\" % output.shape)\n",
    "    \n",
    "    # reshape\n",
    "    output = tf.reshape(output, [-1, num_classes])\n",
    "    \n",
    "    cur_label_one_hot = tf.one_hot(cur_label_ids_padding, 4)\n",
    "    y_real = tf.reshape(cur_label_one_hot, [-1, num_classes])\n",
    "    print(\"output shape: %r, y_real shape: %r\" % (output.shape, y_real.shape))\n",
    "\n",
    "    # 计算损失\n",
    "    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y_real))\n",
    "    \n",
    "    # 计算当前训练集上的精度\n",
    "    print(output.shape, y_real.shape)\n",
    "    correct_pred = tf.equal(tf.argmax(output, 1), tf.argmax(y_real, 1))\n",
    "    \n",
    "    all_chars_num = tf.reshape(inputs, [-1, 1]).get_shape()[0]\n",
    "    all_true_chars_num = tf.reduce_sum(lengths)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) - 1 + tf.divide(all_true_chars_num, all_chars_num)\n",
    "    \n",
    "    return cross_entropy, accuracy\n",
    "\n",
    "\n",
    "def train(data_folder):\n",
    "    # 获取数据\n",
    "    all_data_obj = WSTrainData(data_folder)\n",
    "    # 创建dict和mapping，并将数据转为id序列\n",
    "    feature_dic = create_dic(all_data_obj.features, True, True)\n",
    "    label_dic = create_dic(all_data_obj.labels)\n",
    "    feature_2_id, id_2_feature = create_mapping(feature_dic)\n",
    "    label_2_id, id_2_label = create_mapping(label_dic)\n",
    "    \n",
    "    # 输出batch进行训练\n",
    "    iters = 40000\n",
    "    batch_size = 32\n",
    "    embedding_size = 256\n",
    "    # 创建网络\n",
    "    x, y, embeddings, embeddings_out, lengths = build_input_graph(len(feature_dic), embedding_size)\n",
    "    cross_entropy, accuracy = build_tag_graph(embeddings_out, 512, 128, 4, y, lengths)\n",
    "    \n",
    "    # 创建优化方案\n",
    "    lr = 1e-5\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate=lr).minimize(cross_entropy)\n",
    "    \n",
    "    # 创建session并且初始化\n",
    "    init_op = tf.global_variables_initializer() \n",
    "    \n",
    "    with tf.Session() as sess:\n",
    "        sess.run(init_op)\n",
    "        for i in range(iters):\n",
    "            cur_features, cur_labels = all_data_obj.next_batch(batch_size)\n",
    "#             print(cur_features)\n",
    "#             print(len(cur_features))\n",
    "            cur_feature_ids = data_to_ids([cur_features], mappings=[feature_2_id])[0]\n",
    "#             print(np.array(cur_feature_ids).shape)\n",
    "            cur_label_ids = data_to_ids([cur_labels], mappings=[label_2_id])[0]\n",
    "            # padding\n",
    "            input_info = create_input(cur_feature_ids)\n",
    "            cur_feature_ids_padding = input_info[:-1]\n",
    "            cur_lengths = input_info[-1]\n",
    "            \n",
    "            label_info = create_input(cur_label_ids)\n",
    "            cur_label_ids_padding = label_info[:-1]\n",
    "            cur_label_lengths = label_info[-1]\n",
    "            assert cur_lengths == cur_label_lengths\n",
    "            \n",
    "            sess.run(optimizer, feed_dict={x: cur_feature_ids_padding, y: cur_label_ids_padding, lengths: cur_lengths})\n",
    "            if (i+1)%100 == 0:\n",
    "                cost, cur_accuracy = sess.run([cross_entropy, accuracy], feed_dict={x: cur_feature_ids_padding, y: cur_label_ids_padding, lengths: cur_lengths})\n",
    "                print(\"cost: %f, train accuracy: %f\" % (cost, cur_accuracy))\n",
    "                \n",
    "train_data_folder = \"./data/data/train_pku\"\n",
    "train(train_data_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()\n",
    "a = tf.placeholder(tf.float32, [5, 1])\n",
    "b = tf.placeholder(tf.float32, [3,4,5,6])\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "print(tf.multiply(a,b).shape)\n",
    "print((a*b).shape)\n",
    "a = tf.sequence_mask([1,2,3,4])\n",
    "print(sess.run(a))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mx_py3",
   "language": "python",
   "name": "mx_py3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
